package ingest.blog.icwsm;

import ingest.blog.icwsm.icwsmutils.DatasetUtils;
import ingest.blog.icwsm.icwsmutils.Constants.Language;
import ingest.blog.icwsm.icwsmutils.Constants.SourceType;
import ingest.blog.icwsm.protoutils.ProtoStreamUtils;

import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import com.spinn3r.api.protobuf.ContentApi.Entry;
import com.spinn3r.api.protobuf.ContentApi.PermalinkEntry;


public class ICWSM2011Ingester {

	
	public static void main(String[] args) throws Exception {

//		BlogDB db = BlogDB.getInstance();
		SourceType sourceType = SourceType.valueOf("WEBLOG");
		Language language = Language.valueOf("en");
		String baseDirectory = "/local/corpora/blogs/2011";

		ArrayList<String[]> dates = new ArrayList<String[]>();
		dates.add(new String[] { "01", "13" });
		dates.add(new String[] { "01", "22" });
		dates.add(new String[] { "01", "23" });
		dates.add(new String[] { "01", "27" });
		dates.add(new String[] { "01", "28" });
		dates.add(new String[] { "01", "29" });
		dates.add(new String[] { "02", "11" });
		dates.add(new String[] { "02", "13" });

		for (String[] date : dates) {
			List<String> filenames = DatasetUtils.getFileNamesFor(sourceType,
					language, date[0], date[1], baseDirectory);

			for (String file : filenames) {
				List<Entry> entries = ProtoStreamUtils.read(file);
				// print some information about the entry

				for (Entry entry : entries) {

					// check if we have a permalink entry
					if (entry.hasPermalinkEntry()) {
						PermalinkEntry pe = entry.getPermalinkEntry();
						if (pe.hasContentExtract()) {

							String content = ProtoStreamUtils
									.contentToStringWithoutHTML(pe
											.getContentExtract());

							String la = entry.getPermalinkEntry().getLinkList()
									.get(0).toString().split("\n")[1];
							String cleanURL = la.split("\\:", 2)[1].replaceAll(
									"\"", "").trim();
							URL myUrl = new URL(cleanURL);
							String host = getTopLevelDomain(myUrl.getHost());
							if (host.contains("blogspot")) {
//								Blog i = new Blog();
//								i.setLink(myUrl.toString());
//								i.setLang("en");
//								db.saveEntry(i);
							}
						}
					}
				}
			}

		}
	}

	public static String getTopLevelDomain(String uri) {
		String tld = "";
		String[] parts = uri.split("\\.");
		if (parts.length >= 2) {
			int maxIndex = parts.length - 1;
			tld = parts[maxIndex - 1] + "." + parts[maxIndex];
		}
		return tld;
	}
}
